{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   \n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as Data\n",
    "torch.manual_seed(8) # for reproduce\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import gc\n",
    "import sys\n",
    "sys.setrecursionlimit(50000)\n",
    "import pickle\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.set_default_tensor_type('torch.cuda.FloatTensor')\n",
    "# from tensorboardX import SummaryWriter\n",
    "torch.nn.Module.dump_patches = True\n",
    "import copy\n",
    "import pandas as pd\n",
    "#then import my own modules\n",
    "from AttentiveFP import Fingerprint, Fingerprint_viz, save_smiles_dicts, get_smiles_dicts, get_smiles_array, moltosvg_highlight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import f1_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from rdkit.Chem import rdMolDescriptors, MolSurf\n",
    "# from rdkit.Chem.Draw import SimilarityMaps\n",
    "from rdkit import Chem\n",
    "# from rdkit.Chem import AllChem\n",
    "from rdkit.Chem import QED\n",
    "%matplotlib inline\n",
    "from numpy.polynomial.polynomial import polyfit\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib\n",
    "from IPython.display import SVG, display\n",
    "import seaborn as sns; sns.set(color_codes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of all smiles:  2050\n",
      "not successfully processed smiles:  O=N([O-])C1=C(CN=C1NCCSCc2ncccc2)Cc3ccccc3\n",
      "not successfully processed smiles:  c1(nc(NC(N)=[NH2])sc1)CSCCNC(=[NH]C#N)NC\n",
      "not successfully processed smiles:  Cc1nc(sc1)\\[NH]=C(\\N)N\n",
      "not successfully processed smiles:  s1cc(CSCCN\\C(NC)=[NH]\\C#N)nc1\\[NH]=C(\\N)N\n",
      "not successfully processed smiles:  c1c(c(ncc1)CSCCN\\C(=[NH]\\C#N)NCC)Br\n",
      "not successfully processed smiles:  n1c(csc1\\[NH]=C(\\N)N)c1ccccc1\n",
      "not successfully processed smiles:  n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)N\n",
      "not successfully processed smiles:  n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)NC(C)=O\n",
      "not successfully processed smiles:  n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)N\\C(NC)=[NH]\\C#N\n",
      "not successfully processed smiles:  s1cc(nc1\\[NH]=C(\\N)N)C\n",
      "not successfully processed smiles:  c1(cc(N\\C(=[NH]\\c2cccc(c2)CC)C)ccc1)CC\n",
      "number of successfully processed smiles:  2039\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAC/CAYAAAB+KF5fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEpRJREFUeJzt3X9MVff9x/HX5YeogMDcHXxX1mp3uaTUoc2KyCSS8cOYph3Rbe3cH6urC91SNrcWZzcm6xLTGWXSFkKo3Uy3NfvRLBmGuaC0dHQTSxqdRqP2XnBZiI23tIoXwd4C937/aLjzCtwLn164cHk+EiP3c9735HM+wIvPPefcz7X4fD6fAADTEhPpDgDAfER4AoABwhMADBCeAGCA8AQAA4QnABggPAHAAOEJAAYITwAwQHgCgAHCEwAMEJ4AYIDwBAADcZHuQDhcuzYor3fyxaGWL0/SBx/cmMUe4VaMf2Qx/sHFxFiUlpY47edFRXh6vb6g4TlWg8hh/COL8Q8/XrYDgAHCEwAMEJ4AYIDwBAADUXHBaD4b8Uqe4ZGgNQnxcYrjzxwwpxCeEeYZHtHbF1xBa/LuSVdcAt8qYC5hPgMABghPADBAeAKAAcITAAwQngBggPAEAAOEJwAYIDwBwADhCQAGCE8AMEB4AoABwhMADBCeAGCA8AQAA4QnABggPAHAAOEJAAYITwAwQHgCgAHCEwAMEJ4AYIDwBAADhCcAGCA8AcCAUXjW19crOztb5eXl47YdP35cDz/8sHJzc1VQUKCamhq53e5xdYODg9qzZ48KCwuVm5urLVu26PXXXzfpDgDMummHp9Pp1EsvvaRPf/rT47Z1dXWpoqJCGRkZampq0q5du9Te3q6Kigp5vd6A2srKSrW0tGjHjh168cUXZbPZVFlZqY6ODvOjAYBZEjedYq/Xq+rqan3961+Xw+EYN6Pcv3+/srKy9Nxzzykm5uNctlqteuyxx9Ta2qoHHnhAktTR0aHOzk41NDSorKxMkrRu3Tr19vZq7969KioqCsexAcCMmdbM8+WXX9aVK1f0ox/9aNw2l8uls2fPqry83B+ckrR+/Xqlp6fr6NGj/ra2tjYlJyerpKTE32axWLR582ZdunRJ3d3dJscCALNmyuHZ29urF154QTU1NUpKShq33eFwSJKysrLGbbPb7XI6nf7HTqdTNpstIGQlKTs7O2BfADBXTSk8fT6ffvazn6mwsFClpaUT1vT390uSUlJSxm1LSUnxbx+rnazu1n0BwFw1pXOer776qs6dO6e///3vIWstFsuU2ierC7VtIsuXj58J385qTZ7WPsNhYOgj3fxwJGhNbLyUnLQ4aM3SpQmyfmppOLs26yIx/vgfxj/8Qobn1atXtX//fj3++ONasmSJ/yLRyMiIvF6v3G63EhISlJqaKmniWeP169cDZpqpqamT1kkTz16D+eCDG/J6fZNut1qT1dc3MK19hsOgZ0RvX3AFrVltt2rgxodBa4aGPOobHQ1n12ZVpMYfH2P8g4uJsUxpAjbueaEKXC6XBgYG9Ktf/Up5eXn+f6dOnZLD4VBeXp7q6+v95zpvPbc5xuFwBJwLtdls6unpGXf70ti5TrvdPu0DAYDZFHLmeeedd+p3v/vduPZnn31WQ0ND2rNnjz772c8qIyNDq1atUktLix599FH/xaATJ07I5XJp48aN/ueWlZXpL3/5i9rb2wPOoTY3N2vlypWy2WzhODYAmDEhwzMxMVH5+fnj2pctWyZJAduqqqq0fft2Pfnkk3rkkUfkcrlUW1ur1atXa9OmTf66oqIi5efnq7q6Wv39/crMzFRzc7NOnjypxsbGcBwXAMyoad0kH0pBQYGamppUX1+viooKJSYmqrS0VDt37lRsbKy/zmKxqLGxUQcOHFBdXZ3cbrdsNpsaGhpUXFwczi4BwIyw+Hy+ya+0zBPz/YLRGUdf0Jq8e9KVmBDWv3OzigsWkcX4BzdjF4wAAOMRngBggPAEAAOEJwAYIDwBwADhCQAGCE8AMEB4AoABwhMADMzft60sIJYYiwY9wdcFTYiPUxx/CoFZQ3jOA57h0Sm9hTNuHr+FE5hvmKsAgAHCEwAMEJ4AYIDwBAADhCcAGCA8AcAA4QkABghPADBAeAKAAcITAAwQngBggPAEAAOEJwAYIDwBwADhCQAGCE8AMEB4AoABwhMADBCeAGCA8AQAA4QnABggPAHAAOEJAAb4oO8oYYmxaNAzErQmIT5Ocfy5BMKC8IwSnuFRnXH0Ba3JuyddcQl8y4FwYB4CAAYITwAwQHgCgAHCEwAMEJ4AYCDkpdcTJ07o8OHD+ve//60rV64oJSVFubm5+v73v6/s7OyA2uPHj+v555/XxYsXlZiYqLKyMlVVVWnZsmUBdYODg6qrq1Nra6vcbrdsNpueeOIJlZSUhPfoAGCGhJx5/vGPf9S7776rbdu26aWXXtLTTz+td999V1/72td0+vRpf11XV5cqKiqUkZGhpqYm7dq1S+3t7aqoqJDX6w3YZ2VlpVpaWrRjxw69+OKLstlsqqysVEdHR/iPEABmQMiZ589//nMtX748oK2wsFAlJSX6zW9+o/r6eknS/v37lZWVpeeee04xMR9nstVq1WOPPabW1lY98MADkqSOjg51dnaqoaFBZWVlkqR169apt7dXe/fuVVFRUVgPEABmQsiZ5+3BKUnLli3TXXfdpStXrkiSXC6Xzp49q/Lycn9wStL69euVnp6uo0eP+tva2tqUnJwc8BLdYrFo8+bNunTpkrq7uz/RAQHAbDC6YHT16lU5nU5lZWVJkhwOhyT5H9/KbrfL6XT6HzudTtlstoCQleQ/fzq2LwCYy6b9Xj2fz6fdu3fL6/Vq+/btkqT+/n5JUkpKyrj6lJQUnT9/3v+4v79fK1asmLDu1n1Nx/LlSSFrrNbkae/3k/JdHVJy0uKgNfHxcbNWs3RpgqyfWhq0ZqZEYvzxP4x/+E07PPft26fXXntNv/zlL/X5z38+YJvFYpnwObe3T1YXattkPvjghrxe36TbrdZk9fUNTHu/n9SQZ0QDNz4MWjM8PHs1Q0Me9Y2OBq2ZCZEaf3yM8Q8uJsYypQnY7aYVnnV1dTp06JCqq6u1ZcsWf3tqaqqkiWeN169fD5iRpqamTlonTTx7nYtGvJJnOPgqRkHyHMA8N+XwfP7559XU1KSdO3fqW9/6VsC2sXOdTqdThYWFAdscDofuu+8+/2ObzaZjx47J6/UGnPccO9dpt9unfxQR4Bke0dsXXEFrVtuts9QbALNtSheMGhoa1NjYqB07dug73/nOuO0ZGRlatWqVWlpaAu7pPHHihFwulzZu3OhvKysrk9vtVnt7e8A+mpubtXLlStlsNtNjAYBZE3LmeejQIdXX1+vLX/6yvvSlLwXcGL9o0SLl5ORIkqqqqrR9+3Y9+eSTeuSRR+RyuVRbW6vVq1dr06ZN/ucUFRUpPz9f1dXV6u/vV2Zmppqbm3Xy5Ek1NjbOwCECQPiFDM833njD///Y12PuuOMO/wyyoKBATU1Nqq+vV0VFhRITE1VaWqqdO3cqNjbW/xyLxaLGxkYdOHBAdXV1/rdnNjQ0qLi4OJzHBgAzJmR4/v73v5/yzjZs2KANGzaErEtKSlJNTY1qamqmvG8AmEtYVQkADBCeAGCA8AQAA4QnABggPAHAAOEJAAYITwAwQHgCgAHCEwAMEJ4AYIDwBAADhCcAGCA8AcAA4QkABghPADBAeAKAAcITAAwQngBgYFqf276QhPpcdj6THVjYCM9JhPpcdj6THVjYCM8FxBJj0aBn8tm0JCXExymOkzlASITnAuIZHtUZR1/Qmrx70hWXwI8FEApzDAAwQHgCgAHCEwAMEJ4AYIDwBAADhCcAGCA8AcAA4QkABghPADBAeAKAAcITAAwQngBggPAEAAOEJwAYWJBrj4VaJV5ipXgAwS3I8Ay1SrzESvEAguNlOwAYIDwBwMCCfNmOyfE5R8DURCw8BwcHVVdXp9bWVrndbtlsNj3xxBMqKSmJVJcgPucImKqI/QZUVlbq/PnzqqqqUmZmpv7617+qsrJSTU1NKioqilS3MAVTmZ3Gx8VpeOTjGt/VIQ1NUB9qBjuVuyKYBSNSIhKeHR0d6uzsVENDg8rKyiRJ69atU29vr/bu3Ut4znFTmZ2utlv9NclJizVw48NxNWvvzZBnePJ7wrw+6eTF4HdFMAtGpETkp66trU3JyckBL9EtFos2b96s3bt3q7u7WzabLRJdwywKFcLcLoa5LCLh6XQ6ZbPZFBMT+HorOztbkuRwOKYVnjExlmnVxMXGaOni+KD1oWrCsY+FUrMkIU6jI+PrwzLG8bHyjHiD1iyKi1XsLL20H/VKH42Mzpn+jJnK78hCZTo2EQnP/v5+rVixYlx7SkqKf/t0pKUlhqxZvjwp4HHm/6WEfM7dmWmfaDs1n7xmKvtAaLf//OOTi9ipdotl8rQPtg0A5oKIhGdqauqEs8vr169L+t8MFADmqoiEp81mU09Pj7zewHNVDodDkmS32yPRLQCYsoiEZ1lZmdxut9rb2wPam5ubtXLlSq60A5jzInLBqKioSPn5+aqurlZ/f78yMzPV3NyskydPqrGxMRJdAoBpsfh8voisXHnjxg0dOHBAR48eDXh7ZmlpaSS6AwDTErHwBID5jHcFA4ABwhMADERteA4ODmrPnj0qLCxUbm6utmzZotdffz3S3YoqXV1dys7OnvBfT09PQO3x48f18MMPKzc3VwUFBaqpqZHb7Y5Qz+enK1euaM+ePdq6davuu+8+ZWdnq6ura8LalpYWfeUrX9EXvvAFbdiwQbW1tfJ4POPq3n//fe3atUv5+flas2aNvvnNb+rUqVMzfShRIWqXo2HJu9lTVVWlvLy8gLbMzEz/111dXaqoqFBJSYl++MMf6r333lNtba0cDof+8Ic/jFvjABP773//qyNHjignJ0fr1q0bd6vfmMOHD+vHP/6xtm7dqp/+9Kfq6elRbW2tLl++rLq6On+dx+PRtm3bNDQ0pN27dys1NVW//e1vtW3bNv3pT39STk7ObB3a/OSLQv/4xz98drvdd+zYMX+b1+v1feMb3/Bt2rQpgj2LLm+99ZbPbrf72tragtZ99atf9ZWXl/tGR0f9bf/61798drvdd+TIkZnuZtS4dfza2tp8drvd99ZbbwXUjIyM+NavX+/77ne/G9D+5z//2We3232nT5/2t73yyis+u93uO3funL/N4/H4iouLfdu3b5+ho4geUfknP9iSd5cuXVJ3d3cEe7ewuFwunT17VuXl5QEzzPXr1ys9PV1Hjx6NYO/ml6nM0E+fPq2+vj5t3rw5oP2hhx5SfHx8wHi/9tprstvtuvfee/1tixYt0oMPPqjOzk7duHEjfJ2PQlEZnlNZ8g7hU1NTo5ycHH3xi1/U448/rnPnzvm3jY11VlbWuOfZ7XY5nc5Z6+dCMDaet4/3kiVL9LnPfS5gvJ1O54Rvhc7Oztbo6KguXbo0s52d56LynGe4l7zDxJKTk/Xoo49q7dq1Sk1NVU9Pjw4ePKitW7fqlVde0erVq/1jPdFiLykpKTp//vxsdzuqhRrvW3/2+/v7J62TpGvXrs1QL6NDVIanxJJ3syEnJyfgosL999+v4uJiPfjgg6qrq9PLL7/s3zbZmPO9mBlTHW9+T8xF5ct2lryLHKvVqsLCQp05c0bSx98LaeLZ/vXr1/lehNl0xjvU78nYvjCxqAxPlryLrFvHfezc20TnNh0Ox4TnQmFubEWy28f75s2b6u3tDRhvm8024fn/d955R7Gxsbr77rtntrPzXFSGJ0veRU5fX586Ozu1Zs0aSVJGRoZWrVqllpaWgFA9ceKEXC6XNm7cGKmuRqU1a9bIarXq8OHDAe1/+9vfNDw8HDDeZWVlcjgcunDhgr/to48+0pEjR1RQUKCkJD66I5jYZ5555plIdyLc7rrrLr399tt69dVXlZaWJrfbrYaGBr3xxht69tlntXLlykh3MSo89dRTunDhggYGBvT+++/rn//8p37yk59oYGBA+/fvV3p6uiTpzjvv1KFDh9Td3a2UlBSdPHlSv/jFL5SVlaWnn36am+SnobW1Vd3d3Tpz5oxOnTqlzMxMXb16VZcvX9aKFSsUExOjtLQ0HTx4UNeuXdPixYv15ptvat++fSouLta3v/1t/76ys7N17NgxtbS0yGq16r333tPevXv1zjvvqLa2Vp/5zGcieKRzX9SuqsSSdzPv4MGDOnLkiC5fvqybN28qNTVVa9eu1fe+971xp0befPNN1dfX6+LFi0pMTFRpaal27tzJOc9pGrvd7nZ33HFHwCutw4cP69e//rX+85//KC0tTQ899JB+8IMfaPHixQHP6+vr0759+9TR0SGPx6OcnBw99dRTuv/++2f0OKJB1IYnAMwkXi8BgAHCEwAMEJ4AYIDwBAADhCcAGCA8AcAA4QkABghPADBAeAKAgf8Hx8jJXynQdm0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "task_name = 'BBBP'\n",
    "tasks = ['BBBP']\n",
    "raw_filename = \"../data/BBBP.csv\"\n",
    "feature_filename = raw_filename.replace('.csv','.pickle')\n",
    "filename = raw_filename.replace('.csv','')\n",
    "prefix_filename = raw_filename.split('/')[-1].replace('.csv','')\n",
    "smiles_tasks_df = pd.read_csv(raw_filename)\n",
    "smilesList = smiles_tasks_df.smiles.values\n",
    "print(\"number of all smiles: \",len(smilesList))\n",
    "atom_num_dist = []\n",
    "remained_smiles = []\n",
    "canonical_smiles_list = []\n",
    "for smiles in smilesList:\n",
    "    try:        \n",
    "        mol = Chem.MolFromSmiles(smiles)\n",
    "        atom_num_dist.append(len(mol.GetAtoms()))\n",
    "        remained_smiles.append(smiles)\n",
    "        canonical_smiles_list.append(Chem.MolToSmiles(Chem.MolFromSmiles(smiles), isomericSmiles=True))\n",
    "    except:\n",
    "        print(\"not successfully processed smiles: \", smiles)\n",
    "        pass\n",
    "print(\"number of successfully processed smiles: \", len(remained_smiles))\n",
    "smiles_tasks_df = smiles_tasks_df[smiles_tasks_df[\"smiles\"].isin(remained_smiles)]\n",
    "# print(smiles_tasks_df)\n",
    "smiles_tasks_df['cano_smiles'] =canonical_smiles_list\n",
    "assert canonical_smiles_list[8]==Chem.MolToSmiles(Chem.MolFromSmiles(smiles_tasks_df['cano_smiles'][8]), isomericSmiles=True)\n",
    "\n",
    "plt.figure(figsize=(5, 3))\n",
    "sns.set(font_scale=1.5)\n",
    "ax = sns.distplot(atom_num_dist, bins=28, kde=False)\n",
    "plt.tight_layout()\n",
    "# plt.savefig(\"atom_num_dist_\"+prefix_filename+\".png\",dpi=200)\n",
    "plt.show()\n",
    "plt.close()\n",
    "\n",
    "# print(len([i for i in atom_num_dist if i<51]),len([i for i in atom_num_dist if i>50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed = 188\n",
    "random_seed = int(time.time())\n",
    "start_time = str(time.ctime()).replace(':','-').replace(' ','_')\n",
    "start = time.time()\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 800\n",
    "p_dropout = 0.1\n",
    "fingerprint_dim = 150\n",
    "\n",
    "radius = 3\n",
    "T = 2\n",
    "weight_decay = 2.9 # also known as l2_regularization_lambda\n",
    "learning_rate = 3.5\n",
    "per_task_output_units_num = 2 # for classification model with 2 classes\n",
    "output_units_num = len(tasks) * per_task_output_units_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BBBP</th>\n",
       "      <th>smiles</th>\n",
       "      <th>cano_smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>944</th>\n",
       "      <td>0</td>\n",
       "      <td>CC(C)C1OC(=O)C2=CCCN2C(=O)c3coc(CC(=O)CC(O)\\C=...</td>\n",
       "      <td>C/C1=C/C(O)CC(=O)Cc2nc(co2)C(=O)N2CCC=C2C(=O)O...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     BBBP                                             smiles  \\\n",
       "944     0  CC(C)C1OC(=O)C2=CCCN2C(=O)c3coc(CC(=O)CC(O)\\C=...   \n",
       "\n",
       "                                           cano_smiles  \n",
       "944  C/C1=C/C(O)CC(=O)Cc2nc(co2)C(=O)N2CCC=C2C(=O)O...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smilesList = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())<101]\n",
    "uncovered = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())>100]\n",
    "\n",
    "smiles_tasks_df = smiles_tasks_df[~smiles_tasks_df[\"cano_smiles\"].isin(uncovered)]\n",
    "\n",
    "if os.path.isfile(feature_filename):\n",
    "    feature_dicts = pickle.load(open(feature_filename, \"rb\" ))\n",
    "else:\n",
    "    feature_dicts = save_smiles_dicts(smilesList,filename)\n",
    "# feature_dicts = get_smiles_dicts(smilesList)\n",
    "\n",
    "remained_df = smiles_tasks_df[smiles_tasks_df[\"cano_smiles\"].isin(feature_dicts['smiles_to_atom_mask'].keys())]\n",
    "uncovered_df = smiles_tasks_df.drop(remained_df.index)\n",
    "uncovered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = []\n",
    "for i,task in enumerate(tasks):    \n",
    "    negative_df = remained_df[remained_df[task] == 0][[\"smiles\",task]]\n",
    "    positive_df = remained_df[remained_df[task] == 1][[\"smiles\",task]]\n",
    "    weights.append([(positive_df.shape[0]+negative_df.shape[0])/negative_df.shape[0],\\\n",
    "                    (positive_df.shape[0]+negative_df.shape[0])/positive_df.shape[0]])\n",
    "\n",
    "test_df = remained_df.sample(frac=1/10, random_state=random_seed) # test set\n",
    "training_data = remained_df.drop(test_df.index) # training data\n",
    "\n",
    "# training data is further divided into validation set and train set\n",
    "valid_df = training_data.sample(frac=1/9, random_state=random_seed) # validation set\n",
    "train_df = training_data.drop(valid_df.index) # train set\n",
    "train_df = train_df.reset_index(drop=True)\n",
    "valid_df = valid_df.reset_index(drop=True)\n",
    "test_df = test_df.reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "649206\n",
      "atom_fc.weight torch.Size([150, 39])\n",
      "atom_fc.bias torch.Size([150])\n",
      "neighbor_fc.weight torch.Size([150, 49])\n",
      "neighbor_fc.bias torch.Size([150])\n",
      "GRUCell.0.weight_ih torch.Size([450, 150])\n",
      "GRUCell.0.weight_hh torch.Size([450, 150])\n",
      "GRUCell.0.bias_ih torch.Size([450])\n",
      "GRUCell.0.bias_hh torch.Size([450])\n",
      "GRUCell.1.weight_ih torch.Size([450, 150])\n",
      "GRUCell.1.weight_hh torch.Size([450, 150])\n",
      "GRUCell.1.bias_ih torch.Size([450])\n",
      "GRUCell.1.bias_hh torch.Size([450])\n",
      "GRUCell.2.weight_ih torch.Size([450, 150])\n",
      "GRUCell.2.weight_hh torch.Size([450, 150])\n",
      "GRUCell.2.bias_ih torch.Size([450])\n",
      "GRUCell.2.bias_hh torch.Size([450])\n",
      "align.0.weight torch.Size([1, 300])\n",
      "align.0.bias torch.Size([1])\n",
      "align.1.weight torch.Size([1, 300])\n",
      "align.1.bias torch.Size([1])\n",
      "align.2.weight torch.Size([1, 300])\n",
      "align.2.bias torch.Size([1])\n",
      "attend.0.weight torch.Size([150, 150])\n",
      "attend.0.bias torch.Size([150])\n",
      "attend.1.weight torch.Size([150, 150])\n",
      "attend.1.bias torch.Size([150])\n",
      "attend.2.weight torch.Size([150, 150])\n",
      "attend.2.bias torch.Size([150])\n",
      "mol_GRUCell.weight_ih torch.Size([450, 150])\n",
      "mol_GRUCell.weight_hh torch.Size([450, 150])\n",
      "mol_GRUCell.bias_ih torch.Size([450])\n",
      "mol_GRUCell.bias_hh torch.Size([450])\n",
      "mol_align.weight torch.Size([1, 300])\n",
      "mol_align.bias torch.Size([1])\n",
      "mol_attend.weight torch.Size([150, 150])\n",
      "mol_attend.bias torch.Size([150])\n",
      "output.weight torch.Size([2, 150])\n",
      "output.bias torch.Size([2])\n"
     ]
    }
   ],
   "source": [
    "x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array([smilesList[0]],feature_dicts)\n",
    "num_atom_features = x_atom.shape[-1]\n",
    "num_bond_features = x_bonds.shape[-1]\n",
    "\n",
    "loss_function = [nn.CrossEntropyLoss(torch.Tensor(weight),reduction='mean') for weight in weights]\n",
    "model = Fingerprint(radius, T, num_atom_features,num_bond_features,\n",
    "            fingerprint_dim, output_units_num, p_dropout)\n",
    "model.cuda()\n",
    "# tensorboard = SummaryWriter(log_dir=\"runs/\"+start_time+\"_\"+prefix_filename+\"_\"+str(fingerprint_dim)+\"_\"+str(p_dropout))\n",
    "\n",
    "# optimizer = optim.Adam(model.parameters(), learning_rate, weight_decay=weight_decay)\n",
    "optimizer = optim.Adam(model.parameters(), 10**-learning_rate, weight_decay=10**-weight_decay)\n",
    "model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "params = sum([np.prod(p.size()) for p in model_parameters])\n",
    "print(params)\n",
    "for name, param in model.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, param.data.shape)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, dataset, optimizer, loss_function):\n",
    "    model.train()\n",
    "    np.random.seed(epoch)\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    #shuffle them\n",
    "    np.random.shuffle(valList)\n",
    "    batch_list = []\n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, train_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[train_batch,:]\n",
    "        smiles_list = batch_df.cano_smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "#         print(torch.Tensor(x_atom).size(),torch.Tensor(x_bonds).size(),torch.cuda.LongTensor(x_atom_index).size(),torch.cuda.LongTensor(x_bond_index).size(),torch.Tensor(x_mask).size())\n",
    "        \n",
    "        model.zero_grad()\n",
    "        # Step 4. Compute your loss function. (Again, Torch wants the target wrapped in a variable)\n",
    "        loss = 0.0\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where(y_val != -1)[0]\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "\n",
    "            loss += loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "        # Step 5. Do the backward pass and update the gradient\n",
    "#             print(y_val,y_pred,validInds,y_val_adjust,y_pred_adjust)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "def eval(model, dataset):\n",
    "    model.eval()\n",
    "    y_val_list = {}\n",
    "    y_pred_list = {}\n",
    "    losses_list = []\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    batch_list = []\n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, test_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[test_batch,:]\n",
    "        smiles_list = batch_df.cano_smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "        atom_pred = atoms_prediction.data[:,:,1].unsqueeze(2).cpu().numpy()\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where((y_val=='0') | (y_val=='1'))[0]\n",
    "#             print(validInds)\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "#             print(validInds)\n",
    "            loss = loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "#             print(y_pred_adjust)\n",
    "            y_pred_adjust = F.softmax(y_pred_adjust,dim=-1).data.cpu().numpy()[:,1]\n",
    "            losses_list.append(loss.cpu().detach().numpy())\n",
    "            try:\n",
    "                y_val_list[i].extend(y_val_adjust)\n",
    "                y_pred_list[i].extend(y_pred_adjust)\n",
    "            except:\n",
    "                y_val_list[i] = []\n",
    "                y_pred_list[i] = []\n",
    "                y_val_list[i].extend(y_val_adjust)\n",
    "                y_pred_list[i].extend(y_pred_adjust)\n",
    "#             print(y_val,y_pred,validInds,y_val_adjust,y_pred_adjust)            \n",
    "    test_roc = [roc_auc_score(y_val_list[i], y_pred_list[i]) for i in range(len(tasks))]\n",
    "    test_prc = [auc(precision_recall_curve(y_val_list[i], y_pred_list[i])[1],precision_recall_curve(y_val_list[i], y_pred_list[i])[0]) for i in range(len(tasks))]\n",
    "#     test_prc = auc(recall, precision)\n",
    "    test_precision = [precision_score(y_val_list[i],\n",
    "                                     (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "    test_recall = [recall_score(y_val_list[i],\n",
    "                               (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "    test_loss = np.array(losses_list).mean()\n",
    "    \n",
    "    return test_roc, test_prc, test_precision, test_recall, test_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pytorch/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/pytorch/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:\t0\n",
      "train_roc:[0.6979398729869248]\n",
      "valid_roc:[0.7282221090168111]\n",
      "\n",
      "EPOCH:\t1\n",
      "train_roc:[0.7548591468645403]\n",
      "valid_roc:[0.7869332654100867]\n",
      "\n",
      "EPOCH:\t2\n",
      "train_roc:[0.7948851199363856]\n",
      "valid_roc:[0.8176260825267447]\n",
      "\n",
      "EPOCH:\t3\n",
      "train_roc:[0.8047903247066144]\n",
      "valid_roc:[0.8020886398369842]\n",
      "\n",
      "EPOCH:\t4\n",
      "train_roc:[0.8104754455073284]\n",
      "valid_roc:[0.7957208354559347]\n",
      "\n",
      "EPOCH:\t5\n",
      "train_roc:[0.8156398083783684]\n",
      "valid_roc:[0.8057819663779928]\n",
      "\n",
      "EPOCH:\t6\n",
      "train_roc:[0.8221114681268056]\n",
      "valid_roc:[0.8106214977075905]\n",
      "\n",
      "EPOCH:\t7\n",
      "train_roc:[0.8258583322349159]\n",
      "valid_roc:[0.8116403464085583]\n",
      "\n",
      "EPOCH:\t8\n",
      "train_roc:[0.8309859611441117]\n",
      "valid_roc:[0.8183902190524707]\n",
      "\n",
      "EPOCH:\t9\n",
      "train_roc:[0.8212579437192488]\n",
      "valid_roc:[0.8011971472236372]\n",
      "\n",
      "EPOCH:\t10\n",
      "train_roc:[0.8343179475655108]\n",
      "valid_roc:[0.8245033112582781]\n",
      "\n",
      "EPOCH:\t11\n",
      "train_roc:[0.8379286799326688]\n",
      "valid_roc:[0.8306164034640855]\n",
      "\n",
      "EPOCH:\t12\n",
      "train_roc:[0.8437607365807596]\n",
      "valid_roc:[0.8313805399898114]\n",
      "\n",
      "EPOCH:\t13\n",
      "train_roc:[0.8443247109361327]\n",
      "valid_roc:[0.8278145695364238]\n",
      "\n",
      "EPOCH:\t14\n",
      "train_roc:[0.8469220181206472]\n",
      "valid_roc:[0.8315078960774326]\n",
      "\n",
      "EPOCH:\t15\n",
      "train_roc:[0.8494977170923124]\n",
      "valid_roc:[0.8390219052470708]\n",
      "\n",
      "EPOCH:\t16\n",
      "train_roc:[0.8532899584474066]\n",
      "valid_roc:[0.8374936321956189]\n",
      "\n",
      "EPOCH:\t17\n",
      "train_roc:[0.8556128413287323]\n",
      "valid_roc:[0.8400407539480387]\n",
      "\n",
      "EPOCH:\t18\n",
      "train_roc:[0.8594612640372354]\n",
      "valid_roc:[0.8395313295975547]\n",
      "\n",
      "EPOCH:\t19\n",
      "train_roc:[0.861163991209779]\n",
      "valid_roc:[0.8394039735099338]\n",
      "\n",
      "EPOCH:\t20\n",
      "train_roc:[0.8625490776534346]\n",
      "valid_roc:[0.8495924605196128]\n",
      "\n",
      "EPOCH:\t21\n",
      "train_roc:[0.8627997329224891]\n",
      "valid_roc:[0.8381304126337239]\n",
      "\n",
      "EPOCH:\t22\n",
      "train_roc:[0.8653775927154393]\n",
      "valid_roc:[0.8513754457463067]\n",
      "\n",
      "EPOCH:\t23\n",
      "train_roc:[0.8674022822594412]\n",
      "valid_roc:[0.848191543555782]\n",
      "\n",
      "EPOCH:\t24\n",
      "train_roc:[0.8688132985585162]\n",
      "valid_roc:[0.8515028018339277]\n",
      "\n",
      "EPOCH:\t25\n",
      "train_roc:[0.8679197989571876]\n",
      "valid_roc:[0.8554508405501784]\n",
      "\n",
      "EPOCH:\t26\n",
      "train_roc:[0.8667778049080895]\n",
      "valid_roc:[0.8458991339786042]\n",
      "\n",
      "EPOCH:\t27\n",
      "train_roc:[0.8705527596929042]\n",
      "valid_roc:[0.8540499235863475]\n",
      "\n",
      "EPOCH:\t28\n",
      "train_roc:[0.8715532199478376]\n",
      "valid_roc:[0.847809475292919]\n",
      "\n",
      "EPOCH:\t29\n",
      "train_roc:[0.8727027768714332]\n",
      "valid_roc:[0.8550687722873153]\n",
      "\n",
      "EPOCH:\t30\n",
      "train_roc:[0.8748506332286774]\n",
      "valid_roc:[0.8563423331635251]\n",
      "\n",
      "EPOCH:\t31\n",
      "train_roc:[0.8611359005330746]\n",
      "valid_roc:[0.8211920529801325]\n",
      "\n",
      "EPOCH:\t32\n",
      "train_roc:[0.875116414246727]\n",
      "valid_roc:[0.857615894039735]\n",
      "\n",
      "EPOCH:\t33\n",
      "train_roc:[0.8760001901522732]\n",
      "valid_roc:[0.8554508405501783]\n",
      "\n",
      "EPOCH:\t34\n",
      "train_roc:[0.876659240644184]\n",
      "valid_roc:[0.8583800305654612]\n",
      "\n",
      "EPOCH:\t35\n",
      "train_roc:[0.8794920773487588]\n",
      "valid_roc:[0.8568517575140092]\n",
      "\n",
      "EPOCH:\t36\n",
      "train_roc:[0.8769747205517873]\n",
      "valid_roc:[0.8485736118186449]\n",
      "\n",
      "EPOCH:\t37\n",
      "train_roc:[0.879150667585736]\n",
      "valid_roc:[0.8550687722873154]\n",
      "\n",
      "EPOCH:\t38\n",
      "train_roc:[0.8815470183907499]\n",
      "valid_roc:[0.8634742740703005]\n",
      "\n",
      "EPOCH:\t39\n",
      "train_roc:[0.8813179713345448]\n",
      "valid_roc:[0.8618186449312277]\n",
      "\n",
      "EPOCH:\t40\n",
      "train_roc:[0.8826403939609367]\n",
      "valid_roc:[0.8601630157921548]\n",
      "\n",
      "EPOCH:\t41\n",
      "train_roc:[0.884691013360358]\n",
      "valid_roc:[0.8605450840550178]\n",
      "\n",
      "EPOCH:\t42\n",
      "train_roc:[0.8838029158122418]\n",
      "valid_roc:[0.8526490066225165]\n",
      "\n",
      "EPOCH:\t43\n",
      "train_roc:[0.8865946969124026]\n",
      "valid_roc:[0.8611818644931227]\n",
      "\n",
      "EPOCH:\t44\n",
      "train_roc:[0.8878566165428157]\n",
      "valid_roc:[0.8624554253693326]\n",
      "\n",
      "EPOCH:\t45\n",
      "train_roc:[0.889200647382057]\n",
      "valid_roc:[0.8557055527254203]\n",
      "\n",
      "EPOCH:\t46\n",
      "train_roc:[0.8914327757694145]\n",
      "valid_roc:[0.8651299032093734]\n",
      "\n",
      "EPOCH:\t47\n",
      "train_roc:[0.8898618586952527]\n",
      "valid_roc:[0.8702241467142129]\n",
      "\n",
      "EPOCH:\t48\n",
      "train_roc:[0.8917482556770177]\n",
      "valid_roc:[0.8675496688741722]\n",
      "\n",
      "EPOCH:\t49\n",
      "train_roc:[0.8910416671168377]\n",
      "valid_roc:[0.8676770249617931]\n",
      "\n",
      "EPOCH:\t50\n",
      "train_roc:[0.8921458467934492]\n",
      "valid_roc:[0.8721344880285278]\n",
      "\n",
      "EPOCH:\t51\n",
      "train_roc:[0.8921458467934492]\n",
      "valid_roc:[0.8681864493122772]\n",
      "\n",
      "EPOCH:\t52\n",
      "train_roc:[0.8943931009298013]\n",
      "valid_roc:[0.8723892002037698]\n",
      "\n",
      "EPOCH:\t53\n",
      "train_roc:[0.8961606527408937]\n",
      "valid_roc:[0.8664034640855833]\n",
      "\n",
      "EPOCH:\t54\n",
      "train_roc:[0.8954691899297084]\n",
      "valid_roc:[0.8601630157921548]\n",
      "\n",
      "EPOCH:\t55\n",
      "train_roc:[0.8969601566163268]\n",
      "valid_roc:[0.8646204788588894]\n",
      "\n",
      "EPOCH:\t56\n",
      "train_roc:[0.8975781515038235]\n",
      "valid_roc:[0.8672949566989302]\n",
      "\n",
      "EPOCH:\t57\n",
      "train_roc:[0.8972453850259405]\n",
      "valid_roc:[0.8704788588894549]\n",
      "\n",
      "EPOCH:\t58\n",
      "train_roc:[0.8985310736904881]\n",
      "valid_roc:[0.8664034640855832]\n",
      "\n",
      "EPOCH:\t59\n",
      "train_roc:[0.8999075168490039]\n",
      "valid_roc:[0.868695873662761]\n",
      "\n",
      "EPOCH:\t60\n",
      "train_roc:[0.9015022029572999]\n",
      "valid_roc:[0.869205298013245]\n",
      "\n",
      "EPOCH:\t61\n",
      "train_roc:[0.9014071268207621]\n",
      "valid_roc:[0.8764645950076414]\n",
      "\n",
      "EPOCH:\t62\n",
      "train_roc:[0.9009144595677925]\n",
      "valid_roc:[0.8737901171676007]\n",
      "\n",
      "EPOCH:\t63\n",
      "train_roc:[0.9027425143748636]\n",
      "valid_roc:[0.8754457463066735]\n",
      "\n",
      "EPOCH:\t64\n",
      "train_roc:[0.9021353235937916]\n",
      "valid_roc:[0.8679317371370352]\n",
      "\n",
      "EPOCH:\t65\n",
      "train_roc:[0.9040843843928201]\n",
      "valid_roc:[0.8684411614875193]\n",
      "\n",
      "EPOCH:\t66\n",
      "train_roc:[0.905074040541329]\n",
      "valid_roc:[0.8810494141619969]\n",
      "\n",
      "EPOCH:\t67\n",
      "train_roc:[0.9060464101195582]\n",
      "valid_roc:[0.8711156393275599]\n",
      "\n",
      "EPOCH:\t68\n",
      "train_roc:[0.9039763433285724]\n",
      "valid_roc:[0.8763372389200204]\n",
      "\n",
      "EPOCH:\t69\n",
      "train_roc:[0.9073191338563962]\n",
      "valid_roc:[0.8760825267447784]\n",
      "\n",
      "EPOCH:\t70\n",
      "train_roc:[0.9081056728041195]\n",
      "valid_roc:[0.8753183902190524]\n",
      "\n",
      "EPOCH:\t71\n",
      "train_roc:[0.9091082938803381]\n",
      "valid_roc:[0.8758278145695364]\n",
      "\n",
      "EPOCH:\t72\n",
      "train_roc:[0.9086264307337935]\n",
      "valid_roc:[0.8726439123790116]\n",
      "\n",
      "EPOCH:\t73\n",
      "train_roc:[0.9085551236313898]\n",
      "valid_roc:[0.8686958736627611]\n",
      "\n",
      "EPOCH:\t74\n",
      "train_roc:[0.9122479672073762]\n",
      "valid_roc:[0.8768466632705043]\n",
      "\n",
      "EPOCH:\t75\n",
      "train_roc:[0.907139785689745]\n",
      "valid_roc:[0.8638563423331636]\n",
      "\n",
      "EPOCH:\t76\n",
      "train_roc:[0.910795895303887]\n",
      "valid_roc:[0.879648497198166]\n",
      "\n",
      "EPOCH:\t77\n",
      "train_roc:[0.9131598337896268]\n",
      "valid_roc:[0.8801579215486501]\n",
      "\n",
      "EPOCH:\t78\n",
      "train_roc:[0.9130215412273898]\n",
      "valid_roc:[0.8771013754457463]\n",
      "\n",
      "EPOCH:\t79\n",
      "train_roc:[0.9118633410186545]\n",
      "valid_roc:[0.879393785022924]\n",
      "\n",
      "EPOCH:\t80\n",
      "train_roc:[0.9145730109099867]\n",
      "valid_roc:[0.8750636780438106]\n",
      "\n",
      "EPOCH:\t81\n",
      "train_roc:[0.9141451682955658]\n",
      "valid_roc:[0.8793937850229242]\n",
      "\n",
      "EPOCH:\t82\n",
      "train_roc:[0.9137432555365643]\n",
      "valid_roc:[0.8692052980132451]\n",
      "\n",
      "EPOCH:\t83\n",
      "train_roc:[0.9165890571688488]\n",
      "valid_roc:[0.8776107997962302]\n",
      "\n",
      "EPOCH:\t84\n",
      "train_roc:[0.9190826449316857]\n",
      "valid_roc:[0.8821956189505858]\n",
      "\n",
      "EPOCH:\t85\n",
      "train_roc:[0.9198907920922584]\n",
      "valid_roc:[0.880030565461029]\n",
      "\n",
      "EPOCH:\t86\n",
      "train_roc:[0.9205347168351747]\n",
      "valid_roc:[0.8793937850229241]\n",
      "\n",
      "EPOCH:\t87\n",
      "train_roc:[0.919837851970777]\n",
      "valid_roc:[0.8820682628629648]\n",
      "\n",
      "EPOCH:\t88\n",
      "train_roc:[0.9191993292810733]\n",
      "valid_roc:[0.8800305654610291]\n",
      "\n",
      "EPOCH:\t89\n",
      "train_roc:[0.9212564311443493]\n",
      "valid_roc:[0.8838512480896588]\n",
      "\n",
      "EPOCH:\t90\n",
      "train_roc:[0.9207767288190897]\n",
      "valid_roc:[0.8757004584819155]\n",
      "\n",
      "EPOCH:\t91\n",
      "train_roc:[0.9148668826047404]\n",
      "valid_roc:[0.8810494141619969]\n",
      "\n",
      "EPOCH:\t92\n",
      "train_roc:[0.9234647904975722]\n",
      "valid_roc:[0.8818135506877229]\n",
      "\n",
      "EPOCH:\t93\n",
      "train_roc:[0.9231406673048292]\n",
      "valid_roc:[0.879903209373408]\n",
      "\n",
      "EPOCH:\t94\n",
      "train_roc:[0.9236765509834978]\n",
      "valid_roc:[0.8795211411105451]\n",
      "\n",
      "EPOCH:\t95\n",
      "train_roc:[0.9266476802503094]\n",
      "valid_roc:[0.8869077941925624]\n",
      "\n",
      "EPOCH:\t96\n",
      "train_roc:[0.9204612489114864]\n",
      "valid_roc:[0.8809220580743761]\n",
      "\n",
      "EPOCH:\t97\n",
      "train_roc:[0.9269415519450633]\n",
      "valid_roc:[0.8897096281202241]\n",
      "\n",
      "EPOCH:\t98\n",
      "train_roc:[0.9283633723505629]\n",
      "valid_roc:[0.8846153846153847]\n",
      "\n",
      "EPOCH:\t99\n",
      "train_roc:[0.9272030113205426]\n",
      "valid_roc:[0.8846153846153846]\n",
      "\n",
      "EPOCH:\t100\n",
      "train_roc:[0.9285470421597841]\n",
      "valid_roc:[0.8888181355068772]\n",
      "\n",
      "EPOCH:\t101\n",
      "train_roc:[0.9293335811075074]\n",
      "valid_roc:[0.890855832908813]\n",
      "\n",
      "EPOCH:\t102\n",
      "train_roc:[0.9296209703384062]\n",
      "valid_roc:[0.890983188996434]\n",
      "\n",
      "EPOCH:\t103\n",
      "train_roc:[0.9313280191535199]\n",
      "valid_roc:[0.8899643402954661]\n",
      "\n",
      "EPOCH:\t104\n",
      "train_roc:[0.9305004246013826]\n",
      "valid_roc:[0.8913652572592969]\n",
      "\n",
      "EPOCH:\t105\n",
      "train_roc:[0.9290116187360491]\n",
      "valid_roc:[0.8906011207335711]\n",
      "\n",
      "EPOCH:\t106\n",
      "train_roc:[0.9260512935756624]\n",
      "valid_roc:[0.868823229750382]\n",
      "\n",
      "EPOCH:\t107\n",
      "train_roc:[0.9070684785873414]\n",
      "valid_roc:[0.8773560876209883]\n",
      "\n",
      "EPOCH:\t108\n",
      "train_roc:[0.9265871772543308]\n",
      "valid_roc:[0.8771013754457464]\n",
      "\n",
      "EPOCH:\t109\n",
      "train_roc:[0.9311443493442988]\n",
      "valid_roc:[0.8856342333163526]\n",
      "\n",
      "EPOCH:\t110\n",
      "train_roc:[0.9299710233865686]\n",
      "valid_roc:[0.8911105450840551]\n",
      "\n",
      "EPOCH:\t111\n",
      "train_roc:[0.9327736085931541]\n",
      "valid_roc:[0.890728476821192]\n",
      "\n",
      "EPOCH:\t112\n",
      "train_roc:[0.9322182775229211]\n",
      "valid_roc:[0.8835965359144167]\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:\t113\n",
      "train_roc:[0.9326504417799116]\n",
      "valid_roc:[0.8886907794192562]\n",
      "\n",
      "EPOCH:\t114\n",
      "train_roc:[0.9339663819424486]\n",
      "valid_roc:[0.8935303107488538]\n",
      "\n",
      "EPOCH:\t115\n",
      "train_roc:[0.9354789568419164]\n",
      "valid_roc:[0.8942944472745797]\n",
      "\n",
      "EPOCH:\t116\n",
      "train_roc:[0.9343596514163103]\n",
      "valid_roc:[0.8906011207335711]\n",
      "\n",
      "EPOCH:\t117\n",
      "train_roc:[0.934800458958441]\n",
      "valid_roc:[0.8944218033622007]\n",
      "\n",
      "EPOCH:\t118\n",
      "train_roc:[0.9356107669402987]\n",
      "valid_roc:[0.8930208863983697]\n",
      "\n",
      "EPOCH:\t119\n",
      "train_roc:[0.9344763357656978]\n",
      "valid_roc:[0.8842333163525216]\n",
      "\n",
      "EPOCH:\t120\n",
      "train_roc:[0.9342775402074821]\n",
      "valid_roc:[0.8881813550687723]\n",
      "\n",
      "EPOCH:\t121\n",
      "train_roc:[0.9347874940307312]\n",
      "valid_roc:[0.8835965359144168]\n",
      "\n",
      "EPOCH:\t122\n",
      "train_roc:[0.9371190201971966]\n",
      "valid_roc:[0.8930208863983699]\n",
      "\n",
      "EPOCH:\t123\n",
      "train_roc:[0.9366609260847861]\n",
      "valid_roc:[0.8945491594498217]\n",
      "\n",
      "EPOCH:\t124\n",
      "train_roc:[0.9383420450444806]\n",
      "valid_roc:[0.8926388181355068]\n",
      "\n",
      "EPOCH:\t125\n",
      "train_roc:[0.9382123957673831]\n",
      "valid_roc:[0.8940397350993378]\n",
      "\n",
      "EPOCH:\t126\n",
      "train_roc:[0.9392539116267311]\n",
      "valid_roc:[0.8953132959755477]\n",
      "\n",
      "EPOCH:\t127\n",
      "train_roc:[0.934424476054859]\n",
      "valid_roc:[0.8894549159449822]\n",
      "\n",
      "EPOCH:\t128\n",
      "train_roc:[0.9388325514761652]\n",
      "valid_roc:[0.8990066225165563]\n",
      "\n",
      "EPOCH:\t129\n",
      "train_roc:[0.9387634051950466]\n",
      "valid_roc:[0.8922567498726439]\n",
      "\n",
      "EPOCH:\t130\n",
      "train_roc:[0.9398135643395342]\n",
      "valid_roc:[0.8941670911869587]\n",
      "\n",
      "EPOCH:\t131\n",
      "train_roc:[0.940103114391718]\n",
      "valid_roc:[0.8928935303107489]\n",
      "\n",
      "EPOCH:\t132\n",
      "train_roc:[0.935593480370019]\n",
      "valid_roc:[0.8959500764136525]\n",
      "\n",
      "EPOCH:\t133\n",
      "train_roc:[0.9411165395743615]\n",
      "valid_roc:[0.8976057055527253]\n",
      "\n",
      "EPOCH:\t134\n",
      "train_roc:[0.9362201185426556]\n",
      "valid_roc:[0.8856342333163526]\n",
      "\n",
      "EPOCH:\t135\n",
      "train_roc:[0.9359478550607515]\n",
      "valid_roc:[0.8863983698420784]\n",
      "\n",
      "EPOCH:\t136\n",
      "train_roc:[0.9371341459461913]\n",
      "valid_roc:[0.8940397350993377]\n",
      "\n",
      "EPOCH:\t137\n",
      "train_roc:[0.9359889106651657]\n",
      "valid_roc:[0.8843606724401426]\n",
      "\n",
      "EPOCH:\t138\n",
      "train_roc:[0.9386078260625298]\n",
      "valid_roc:[0.8918746816097809]\n",
      "\n",
      "EPOCH:\t139\n",
      "train_roc:[0.9416848355723044]\n",
      "valid_roc:[0.8953132959755477]\n",
      "\n",
      "EPOCH:\t140\n",
      "train_roc:[0.9428624831726042]\n",
      "valid_roc:[0.8988792664289353]\n",
      "\n",
      "EPOCH:\t141\n",
      "train_roc:[0.9370088183116639]\n",
      "valid_roc:[0.8843606724401426]\n",
      "\n",
      "EPOCH:\t142\n",
      "train_roc:[0.9438240486444088]\n",
      "valid_roc:[0.8984971981660723]\n",
      "\n",
      "EPOCH:\t143\n",
      "train_roc:[0.9445068681704544]\n",
      "valid_roc:[0.8969689251146205]\n",
      "\n",
      "EPOCH:\t144\n",
      "train_roc:[0.9449174242145952]\n",
      "valid_roc:[0.9001528273051451]\n",
      "\n",
      "EPOCH:\t145\n",
      "train_roc:[0.9439904318833502]\n",
      "valid_roc:[0.8964595007641365]\n",
      "\n",
      "EPOCH:\t146\n",
      "train_roc:[0.9412072940683296]\n",
      "valid_roc:[0.8986245542536933]\n",
      "\n",
      "EPOCH:\t147\n",
      "train_roc:[0.9382880245123566]\n",
      "valid_roc:[0.8863983698420784]\n",
      "\n",
      "EPOCH:\t148\n",
      "train_roc:[0.9309822877479272]\n",
      "valid_roc:[0.879903209373408]\n",
      "\n",
      "EPOCH:\t149\n",
      "train_roc:[0.9435453026986496]\n",
      "valid_roc:[0.8939123790117167]\n",
      "\n",
      "EPOCH:\t150\n",
      "train_roc:[0.9401938688856861]\n",
      "valid_roc:[0.8865257259296995]\n",
      "\n",
      "EPOCH:\t151\n",
      "train_roc:[0.944139528552012]\n",
      "valid_roc:[0.8970962812022414]\n",
      "\n",
      "EPOCH:\t152\n",
      "train_roc:[0.9439234464235167]\n",
      "valid_roc:[0.8987519103413143]\n",
      "\n",
      "EPOCH:\t153\n",
      "train_roc:[0.9449736055680042]\n",
      "valid_roc:[0.8941670911869587]\n",
      "\n",
      "EPOCH:\t154\n",
      "train_roc:[0.942914342883443]\n",
      "valid_roc:[0.8932755985736118]\n",
      "\n",
      "EPOCH:\t155\n",
      "train_roc:[0.9393165754439946]\n",
      "valid_roc:[0.8872898624554254]\n",
      "\n",
      "EPOCH:\t156\n",
      "train_roc:[0.942171020361419]\n",
      "valid_roc:[0.8931482424859909]\n",
      "\n",
      "EPOCH:\t157\n",
      "train_roc:[0.9460129606060671]\n",
      "valid_roc:[0.8953132959755477]\n",
      "\n",
      "EPOCH:\t158\n",
      "train_roc:[0.9459632617165132]\n",
      "valid_roc:[0.8970962812022415]\n",
      "\n",
      "EPOCH:\t159\n",
      "train_roc:[0.9470782454995494]\n",
      "valid_roc:[0.8983698420784514]\n",
      "\n",
      "EPOCH:\t160\n",
      "train_roc:[0.946203112879143]\n",
      "valid_roc:[0.8974783494651044]\n",
      "\n",
      "EPOCH:\t161\n",
      "train_roc:[0.9483380043086775]\n",
      "valid_roc:[0.8987519103413144]\n",
      "\n",
      "EPOCH:\t162\n",
      "train_roc:[0.9470307074312804]\n",
      "valid_roc:[0.8934029546612328]\n",
      "\n",
      "EPOCH:\t163\n",
      "train_roc:[0.9489019786640507]\n",
      "valid_roc:[0.9029546612328069]\n",
      "\n",
      "EPOCH:\t164\n",
      "train_roc:[0.948834993204217]\n",
      "valid_roc:[0.90206316861946]\n",
      "\n",
      "EPOCH:\t165\n",
      "train_roc:[0.9464083909012138]\n",
      "valid_roc:[0.900916963830871]\n",
      "\n",
      "EPOCH:\t166\n",
      "train_roc:[0.9496690702202093]\n",
      "valid_roc:[0.8990066225165563]\n",
      "\n",
      "EPOCH:\t167\n",
      "train_roc:[0.9473591522665934]\n",
      "valid_roc:[0.8962047885888944]\n",
      "\n",
      "EPOCH:\t168\n",
      "train_roc:[0.9495242951941174]\n",
      "valid_roc:[0.8986245542536933]\n",
      "\n",
      "EPOCH:\t169\n",
      "train_roc:[0.9503237990695504]\n",
      "valid_roc:[0.8990066225165563]\n",
      "\n",
      "EPOCH:\t170\n",
      "train_roc:[0.949541581764397]\n",
      "valid_roc:[0.9016811003565971]\n",
      "\n",
      "EPOCH:\t171\n",
      "train_roc:[0.9508748084972136]\n",
      "valid_roc:[0.901553744268976]\n",
      "\n",
      "EPOCH:\t172\n",
      "train_roc:[0.9506716912964281]\n",
      "valid_roc:[0.9012990320937341]\n",
      "\n",
      "EPOCH:\t173\n",
      "train_roc:[0.9477567433830251]\n",
      "valid_roc:[0.891492613346918]\n",
      "\n",
      "EPOCH:\t174\n",
      "train_roc:[0.9501768632221734]\n",
      "valid_roc:[0.8964595007641365]\n",
      "\n",
      "EPOCH:\t175\n",
      "train_roc:[0.9517434586537651]\n",
      "valid_roc:[0.9034640855832908]\n",
      "\n",
      "EPOCH:\t176\n",
      "train_roc:[0.9508207879650898]\n",
      "valid_roc:[0.9002801833927662]\n",
      "\n",
      "EPOCH:\t177\n",
      "train_roc:[0.9522339650854497]\n",
      "valid_roc:[0.901426388181355]\n",
      "\n",
      "EPOCH:\t178\n",
      "train_roc:[0.9534764373242981]\n",
      "valid_roc:[0.8991339786041772]\n",
      "\n",
      "EPOCH:\t179\n",
      "train_roc:[0.9537054843805034]\n",
      "valid_roc:[0.902954661232807]\n",
      "\n",
      "EPOCH:\t180\n",
      "train_roc:[0.9520027572079595]\n",
      "valid_roc:[0.8964595007641365]\n",
      "\n",
      "EPOCH:\t181\n",
      "train_roc:[0.9493643944190308]\n",
      "valid_roc:[0.902699949057565]\n",
      "\n",
      "EPOCH:\t182\n",
      "train_roc:[0.9513329026096239]\n",
      "valid_roc:[0.8968415690269995]\n",
      "\n",
      "EPOCH:\t183\n",
      "train_roc:[0.9530745245652968]\n",
      "valid_roc:[0.8991339786041772]\n",
      "\n",
      "EPOCH:\t184\n",
      "train_roc:[0.9518018008284588]\n",
      "valid_roc:[0.8984971981660723]\n",
      "\n",
      "EPOCH:\t185\n",
      "train_roc:[0.9494162541298696]\n",
      "valid_roc:[0.8965868568517575]\n",
      "\n",
      "EPOCH:\t186\n",
      "train_roc:[0.953253872731948]\n",
      "valid_roc:[0.8990066225165563]\n",
      "\n",
      "EPOCH:\t187\n",
      "train_roc:[0.9539993560752571]\n",
      "valid_roc:[0.8954406520631686]\n",
      "\n",
      "EPOCH:\t188\n",
      "train_roc:[0.9552828839185198]\n",
      "valid_roc:[0.9021905247070809]\n",
      "\n",
      "EPOCH:\t189\n",
      "train_roc:[0.9512529522220805]\n",
      "valid_roc:[0.9042282221090168]\n",
      "\n",
      "EPOCH:\t190\n",
      "train_roc:[0.9543731781575542]\n",
      "valid_roc:[0.9023178807947019]\n",
      "\n",
      "EPOCH:\t191\n",
      "train_roc:[0.9562033937859103]\n",
      "valid_roc:[0.9021905247070809]\n",
      "\n",
      "EPOCH:\t192\n",
      "train_roc:[0.9546497632820282]\n",
      "valid_roc:[0.9011716760061131]\n",
      "\n",
      "EPOCH:\t193\n",
      "train_roc:[0.9566550054344655]\n",
      "valid_roc:[0.9083036169128884]\n",
      "\n",
      "EPOCH:\t194\n",
      "train_roc:[0.9570266666954776]\n",
      "valid_roc:[0.9042282221090168]\n",
      "\n",
      "EPOCH:\t195\n",
      "train_roc:[0.9538869933684395]\n",
      "valid_roc:[0.8928935303107488]\n",
      "\n",
      "EPOCH:\t196\n",
      "train_roc:[0.9566182714726214]\n",
      "valid_roc:[0.9052470708099847]\n",
      "\n",
      "EPOCH:\t197\n",
      "train_roc:[0.9541981516334728]\n",
      "valid_roc:[0.9057564951604686]\n",
      "\n",
      "EPOCH:\t198\n",
      "train_roc:[0.9556826358562361]\n",
      "valid_roc:[0.9061385634233315]\n",
      "\n",
      "EPOCH:\t199\n",
      "train_roc:[0.9559376127678609]\n",
      "valid_roc:[0.8953132959755475]\n",
      "\n",
      "EPOCH:\t200\n",
      "train_roc:[0.956912143167375]\n",
      "valid_roc:[0.8988792664289352]\n",
      "\n",
      "EPOCH:\t201\n",
      "train_roc:[0.9577721500387867]\n",
      "valid_roc:[0.9062659195109526]\n",
      "\n",
      "EPOCH:\t202\n",
      "train_roc:[0.9577073254002382]\n",
      "valid_roc:[0.9004075394803872]\n",
      "\n",
      "EPOCH:\t203\n",
      "train_roc:[0.9583080337174553]\n",
      "valid_roc:[0.9053744268976056]\n",
      "\n",
      "EPOCH:\t204\n",
      "train_roc:[0.9567781722477079]\n",
      "valid_roc:[0.9039735099337749]\n",
      "\n",
      "EPOCH:\t205\n",
      "train_roc:[0.9581373288359439]\n",
      "valid_roc:[0.9010443199184921]\n",
      "\n",
      "EPOCH:\t206\n",
      "train_roc:[0.9553866033401976]\n",
      "valid_roc:[0.9026999490575649]\n",
      "\n",
      "EPOCH:\t207\n",
      "train_roc:[0.9569726461633538]\n",
      "valid_roc:[0.9079215486500255]\n",
      "\n",
      "EPOCH:\t208\n",
      "train_roc:[0.9593236197213838]\n",
      "valid_roc:[0.8988792664289352]\n",
      "\n",
      "EPOCH:\t209\n",
      "train_roc:[0.9549890122237661]\n",
      "valid_roc:[0.8875445746306674]\n",
      "\n",
      "EPOCH:\t210\n",
      "train_roc:[0.9603694572233014]\n",
      "valid_roc:[0.901426388181355]\n",
      "\n",
      "EPOCH:\t211\n",
      "train_roc:[0.9585370807736605]\n",
      "valid_roc:[0.8959500764136527]\n",
      "\n",
      "EPOCH:\t212\n",
      "train_roc:[0.9596131697735676]\n",
      "valid_roc:[0.8939123790117167]\n",
      "\n",
      "EPOCH:\t213\n",
      "train_roc:[0.9546432808181733]\n",
      "valid_roc:[0.8956953642384106]\n",
      "\n",
      "EPOCH:\t214\n",
      "train_roc:[0.9598983981831815]\n",
      "valid_roc:[0.8978604177279674]\n",
      "\n",
      "EPOCH:\t215\n",
      "train_roc:[0.9589260286049522]\n",
      "valid_roc:[0.9052470708099847]\n",
      "\n",
      "EPOCH:\t216\n",
      "train_roc:[0.9590059789924955]\n",
      "valid_roc:[0.9060112073357106]\n",
      "\n",
      "EPOCH:\t217\n",
      "train_roc:[0.9598162869743533]\n",
      "valid_roc:[0.9030820173204279]\n",
      "\n",
      "EPOCH:\t218\n",
      "train_roc:[0.9601641792012309]\n",
      "valid_roc:[0.902190524707081]\n",
      "\n",
      "EPOCH:\t219\n",
      "train_roc:[0.9596887985185409]\n",
      "valid_roc:[0.8996434029546612]\n",
      "\n",
      "EPOCH:\t220\n",
      "train_roc:[0.9628716882712781]\n",
      "valid_roc:[0.8968415690269995]\n",
      "\n",
      "EPOCH:\t221\n",
      "train_roc:[0.9606784546670498]\n",
      "valid_roc:[0.8990066225165563]\n",
      "\n",
      "EPOCH:\t222\n",
      "train_roc:[0.9621434914982487]\n",
      "valid_roc:[0.9039735099337749]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_param ={}\n",
    "best_param[\"roc_epoch\"] = 0\n",
    "best_param[\"loss_epoch\"] = 0\n",
    "best_param[\"valid_roc\"] = 0\n",
    "best_param[\"valid_loss\"] = 9e8\n",
    "\n",
    "for epoch in range(epochs):    \n",
    "    train_roc, train_prc, train_precision, train_recall, train_loss = eval(model, train_df)\n",
    "    valid_roc, valid_prc, valid_precision, valid_recall, valid_loss = eval(model, valid_df)\n",
    "    train_roc_mean = np.array(train_roc).mean()\n",
    "    valid_roc_mean = np.array(valid_roc).mean()\n",
    "    \n",
    "#     tensorboard.add_scalars('ROC',{'train_roc':train_roc_mean,'valid_roc':valid_roc_mean},epoch)\n",
    "#     tensorboard.add_scalars('Losses',{'train_losses':train_loss,'valid_losses':valid_loss},epoch)\n",
    "\n",
    "    if valid_roc_mean > best_param[\"valid_roc\"]:\n",
    "        best_param[\"roc_epoch\"] = epoch\n",
    "        best_param[\"valid_roc\"] = valid_roc_mean\n",
    "        if valid_roc_mean > 0.87:\n",
    "             torch.save(model, 'saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(epoch)+'.pt')             \n",
    "    if valid_loss < best_param[\"valid_loss\"]:\n",
    "        best_param[\"loss_epoch\"] = epoch\n",
    "        best_param[\"valid_loss\"] = valid_loss\n",
    "\n",
    "    print(\"EPOCH:\\t\"+str(epoch)+'\\n'\\\n",
    "        +\"train_roc\"+\":\"+str(train_roc)+'\\n'\\\n",
    "        +\"valid_roc\"+\":\"+str(valid_roc)+'\\n'\\\n",
    "#         +\"train_roc_mean\"+\":\"+str(train_roc_mean)+'\\n'\\\n",
    "#         +\"valid_roc_mean\"+\":\"+str(valid_roc_mean)+'\\n'\\\n",
    "        )\n",
    "    if (epoch - best_param[\"roc_epoch\"] >18) and (epoch - best_param[\"loss_epoch\"] >28):        \n",
    "        break\n",
    "        \n",
    "    train(model, train_df, optimizer, loss_function)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch:193\n",
      "test_roc:[0.9311052123552124]\n",
      "test_roc_mean: 0.9311052123552124\n"
     ]
    }
   ],
   "source": [
    "# evaluate model\n",
    "best_model = torch.load('saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(best_param[\"roc_epoch\"])+'.pt')     \n",
    "\n",
    "# best_model_dict = best_model.state_dict()\n",
    "# best_model_wts = copy.deepcopy(best_model_dict)\n",
    "\n",
    "# model.load_state_dict(best_model_wts)\n",
    "# (best_model.align[0].weight == model.align[0].weight).all()\n",
    "\n",
    "test_roc, test_prc, test_precision, test_recall, test_losses = eval(best_model, test_df)\n",
    "\n",
    "print(\"best epoch:\"+str(best_param[\"roc_epoch\"])\n",
    "      +\"\\n\"+\"test_roc:\"+str(test_roc)\n",
    "      +\"\\n\"+\"test_roc_mean:\",str(np.array(test_roc).mean())\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
